# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
from scrapy.exceptions import DropItem
import pymongo


class TextPipeline(object):
    def __init__(self):
        self.limit = 50

    def process_item(self, item, spider):
        if item['text']:
            if len(item['text']) > self.limit:
                item['text'] = item['text'][0:self.limit].rstrip() + '...'
            return item
        else:
            return DropItem('Missing Text')


class MongoDBPipeline(object):
    """
    通用的 MongoDB Pipeline
    """

    def __init__(self, host, port, db):
        self.host = host
        self.port = port
        self.db = db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(host=crawler.settings.get("MONGODB_SERVER"), port=crawler.settings.get("MONGODB_PORT"),
                   db=crawler.settings.get("MONGODB_DB"))

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.host, self.port)
        # set the db to be operated
        self.db_conn = self.client[self.db]

    def process_item(self, item, spider):
        # name = item.__class__.__name__
        name = item.getCollection()
        # 这个name是Collection的名字，相当于表明，这就要求item名字要以表名来命名
        count = self.db_conn[name].find(item.getId()).count()
        if count == 0:
            print('插入数据...')
            self.db_conn[name].insert(dict(item))
        else:
            # print('重复数据，更新! praise_nums %d' % item.get_random_id())
            # self.db_conn[name].update({"url_object_id": item.getId()},
            #  {"$set": {"praise_nums": item.get_random_id()}})
            print('重复数据，忽略')
        return item

    def close_spider(self, spider):
        self.client.close()
